/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package no.ntnu.idi.deid.preprocessor.sentenizer.externaltools;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.HashSet;
import java.util.Iterator;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.AttributeSource;

/**
 * Tokenizes input text into sentences.
 *
 * @lucene.experimental
 * 
 * edited by Hans Moen
 */
public class DocumentToSentences extends Tokenizer {

    /**
     * End of sentence punctuation: .,!?;
     */
    private final static String PUNCTION = ".!?;";
    private final static String SPACE = " \t";
    private final static String NEWLINE = "\n\r";
    //private final static String[] ABBR = {"etc", "kl", "ca", "konf", "pas", "ev", "evt", "vedr", "hø", "ve", "u.s", "dr", "mtp", "rtg", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; // witouth dot at the end (.)
    private HashSet<String> ABBR;
    
    private final StringBuilder buffer = new StringBuilder();
    private int tokenStart = 0, tokenEnd = 0;
    private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
    private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);

    public DocumentToSentences(Reader reader, String abbreviationFile) {
        super(reader);
        ABBR = readVocabulary(abbreviationFile);
    }

    public DocumentToSentences(AttributeSource source, Reader reader) {
        super(source, reader);
    }

    public DocumentToSentences(AttributeSource.AttributeFactory factory, Reader reader) {
        super(factory, reader);
    }

    @Override
    public boolean incrementToken() throws IOException {
        clearAttributes();
        buffer.setLength(0);
        int ci;
        char ch;
        //boolean atBegin = true;
        tokenStart = tokenEnd;
        ci = input.read();
        ch = (char) ci;
        @SuppressWarnings("unused")
		String previousSentence = "empty";
        @SuppressWarnings("unused")
		int newLineCounter = 0; //increase when buffer is empty and newLine occurs. If counter reaches 2, there is a section.

        while (true) {

            if (ci == -1) {
                break;

            } else if (PUNCTION.indexOf(ch) != -1) {
                // Possible end of a sentence
                buffer.append(ch);
                tokenEnd++;
                ci = input.read();
                ch = (char) ci;

                // If it is a dot followed by a space, check for possible abbreviation
                if ((SPACE.indexOf(ch) != -1) && (buffer.substring(buffer.length() - 1, buffer.length()).equals("."))) {

                    // Test to see if it is a abbreviation
                    boolean isAbbr = false;
                    
                    Iterator<String> it = ABBR.iterator();
                    
                    String abbr = "";
                    while (it.hasNext()) {
                        abbr = it.next();
                        if (buffer.length() > abbr.length() && !abbr.trim().isEmpty()) {
                           if (buffer.substring(buffer.length() - abbr.length() - 1, buffer.length()).toLowerCase().endsWith(abbr + ".")) {
                                buffer.append(ch);
                                
                                tokenEnd++;
                                ci = input.read();
                                ch = (char) ci;

                                isAbbr = true;
                                break; // only this for-loop!
                            }
                        }
                    }
                    if (!isAbbr) { // last word is not an abbreviation
                        break;
                    }

                } else if (SPACE.indexOf(ch) != -1) { // Only a space after end char
                    break;

                } else if (PUNCTION.indexOf(ch) != -1) { // A new end char
                    continue;

                } else if (NEWLINE.indexOf(ch) == -1) { // Not a linebreak char
                    continue;

                } else {
                    break;
                }

            } 
            
            else if (NEWLINE.indexOf(ch) != -1) {
            	buffer.append(ch);
            	break;
//            	if(buffer.toString().trim().isEmpty()){
//        			buffer.append(ch);
//        			break;
//            	}
//            	else {
//            		tokenEnd++;
//            		ci = input.read();
//            		ch = (char) ci;
//            		continue;
//				}
                
            } 
            
            else {
                buffer.append(ch);
                //atBegin = false;
                tokenEnd++;
                //pch = ch;
                ci = input.read();
                ch = (char) ci;
                // Two spaces, such as CR, LF
                /*if (Utility.SPACES.indexOf(ch) != -1 && Utility.SPACES.indexOf(pch) != -1) {
                    // buffer.append(ch);
                    tokenEnd++;
                    break;
                 }*/
            }
            
        }
        if (buffer.length() == 0) {
            return false;
        } else {
            termAtt.setEmpty().append(buffer);
            offsetAtt.setOffset(correctOffset(tokenStart), correctOffset(tokenEnd));
            typeAtt.setType("sentence");
            previousSentence = buffer.toString();
            return true;
        }
    }

    @Override
    public void reset() throws IOException {
        super.reset();
        tokenStart = tokenEnd = 0;
    }

//    @Override
//    public void reset(Reader input) throws IOException {
//        super.reset(input);
//        reset();
//    }

    @Override
    public void end() throws IOException {
        // set final offset
        final int finalOffset = correctOffset(tokenEnd);
        offsetAtt.setOffset(finalOffset, finalOffset);
    }

    /**
     * Reads a list of words from a file into a
     * <code>HashSet</code>. All incoming words are transformed to lower case.
     *
     * @param vocabularyfile filename of the file containing the list of words
     * @return a <code>HashSet</code> containing the words
     */
    public static HashSet<String> readVocabulary(String vocabularyfile) {
        return readVocabulary(vocabularyfile, true);
    }

    /**
     * Reads a list of words from a file into a
     * <code>HashSet</code>. All incoming words are optionally transformed to
     * lower case.
     *
     * @param vocabularyfile filename of the file containing the list of words
     * @param lowercase <code>true</code> if all words should be converted to
     * lower case, <code>false</code> original casing should be preserved
     * @return a <code>HashSet</code> containing the words
     */
    public static HashSet<String> readVocabulary(String vocabularyfile, boolean lowercase) {
        HashSet<String> wordSet = new HashSet<String>();
        try {
            BufferedReader fr = new BufferedReader(new FileReader(vocabularyfile));
            String word = fr.readLine();
            // If the word list we're reading from is a FrequencyList
            if (word != null && word.indexOf("\t") > 0) {
                word = word.split("\t")[0];
            }
            while (word != null) {
                // Do not add comments
                if (!word.startsWith("#")) {
                    if (lowercase) {
                        wordSet.add(word.toLowerCase());
                    } else {
                        wordSet.add(word);
                    }
                }
                word = fr.readLine();
                if (word != null && word.indexOf("\t") > 0) {
                    word = word.split("\t")[0];
                }
            }
            fr.close();
        } catch (FileNotFoundException fnfe) {
//            System.err.println("Warning: vocabulary file '" + wordSet + "' not found.");
        } catch (IOException ioe) {
            ioe.printStackTrace();
        }
        return wordSet;
    }

    
    
    /**
     * Testing..
     * Breaks the text into sentences, one sentence at each line
     *
     * @param args
     * @throws Exception
     */
    public static void main(String[] args) throws Exception {

        //String file = "data/test/en/inn/docEN1.txt";
        // NO EPJ
        //String file = "data/epj/WithoutSentenceTags/Original/j1/epj1_00.txt";

        // DE
        String file = "data/epj/WithoutSentenceTags/Original/j1/epj1_00.txt";

        FileReader reader = new FileReader(file);

        //String docText = SentenceTokenize(reader);

        String docText = "";
        DocumentToSentences sentTokenizer = new DocumentToSentences(reader, "abbreviations.txt");

        CharTermAttribute sentCharAttr = sentTokenizer.addAttribute(CharTermAttribute.class);
        while (sentTokenizer.incrementToken()) {
            if (!sentCharAttr.toString().trim().isEmpty()) {
                docText += "<s> " + sentCharAttr.toString().trim() + " </s>\n";
                //System.out.println("$ " + sentCharAttr.toString().trim());
            }
        }
        sentTokenizer.close();

        System.out.println(docText);
    }
}
